/*-------------------------------------------------------------------------
 *
 * dict_snowball.c
 *		Snowball 词典
 *
 * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *	  src/backend/snowball/dict_snowball.c
 *
 *-------------------------------------------------------------------------
 */
#include "postgres.h"

#include "commands/defrem.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_utils.h"

/* 一些平台定义了MAXINT和/或MININT，导致冲突 */
#ifdef MAXINT
#undef MAXINT
#endif
#ifdef MININT
#undef MININT
#endif

/* 现在我们可以包含原始的Snowball header.h */
#include "snowball/libstemmer/header.h"
#include "snowball/libstemmer/stem_ISO_8859_1_basque.h"
#include "snowball/libstemmer/stem_ISO_8859_1_catalan.h"
#include "snowball/libstemmer/stem_ISO_8859_1_danish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_dutch.h"
#include "snowball/libstemmer/stem_ISO_8859_1_english.h"
#include "snowball/libstemmer/stem_ISO_8859_1_finnish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_french.h"
#include "snowball/libstemmer/stem_ISO_8859_1_german.h"
#include "snowball/libstemmer/stem_ISO_8859_1_indonesian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_irish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_italian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_norwegian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_porter.h"
#include "snowball/libstemmer/stem_ISO_8859_1_portuguese.h"
#include "snowball/libstemmer/stem_ISO_8859_1_spanish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_swedish.h"
#include "snowball/libstemmer/stem_ISO_8859_2_hungarian.h"
#include "snowball/libstemmer/stem_ISO_8859_2_romanian.h"
#include "snowball/libstemmer/stem_KOI8_R_russian.h"
#include "snowball/libstemmer/stem_UTF_8_arabic.h"
#include "snowball/libstemmer/stem_UTF_8_armenian.h"
#include "snowball/libstemmer/stem_UTF_8_basque.h"
#include "snowball/libstemmer/stem_UTF_8_catalan.h"
#include "snowball/libstemmer/stem_UTF_8_danish.h"
#include "snowball/libstemmer/stem_UTF_8_dutch.h"
#include "snowball/libstemmer/stem_UTF_8_english.h"
#include "snowball/libstemmer/stem_UTF_8_finnish.h"
#include "snowball/libstemmer/stem_UTF_8_french.h"
#include "snowball/libstemmer/stem_UTF_8_german.h"
#include "snowball/libstemmer/stem_UTF_8_greek.h"
#include "snowball/libstemmer/stem_UTF_8_hindi.h"
#include "snowball/libstemmer/stem_UTF_8_hungarian.h"
#include "snowball/libstemmer/stem_UTF_8_indonesian.h"
#include "snowball/libstemmer/stem_UTF_8_irish.h"
#include "snowball/libstemmer/stem_UTF_8_italian.h"
#include "snowball/libstemmer/stem_UTF_8_lithuanian.h"
#include "snowball/libstemmer/stem_UTF_8_nepali.h"
#include "snowball/libstemmer/stem_UTF_8_norwegian.h"
#include "snowball/libstemmer/stem_UTF_8_porter.h"
#include "snowball/libstemmer/stem_UTF_8_portuguese.h"
#include "snowball/libstemmer/stem_UTF_8_romanian.h"
#include "snowball/libstemmer/stem_UTF_8_russian.h"
#include "snowball/libstemmer/stem_UTF_8_serbian.h"
#include "snowball/libstemmer/stem_UTF_8_spanish.h"
#include "snowball/libstemmer/stem_UTF_8_swedish.h"
#include "snowball/libstemmer/stem_UTF_8_tamil.h"
#include "snowball/libstemmer/stem_UTF_8_turkish.h"
#include "snowball/libstemmer/stem_UTF_8_yiddish.h"

PG_MODULE_MAGIC;

PG_FUNCTION_INFO_V1(dsnowball_init);

PG_FUNCTION_INFO_V1(dsnowball_lexize);

/* 支持的模块列表 */
typedef struct stemmer_module
{
	const char *name;
	pg_enc		enc;
	struct SN_env *(*create) (void);
	void		(*close) (struct SN_env *);
	int			(*stem) (struct SN_env *);
} stemmer_module;

/* 参数：词干提取器名称，PG编码代码，Snowball的编码名称 */
#define STEMMER_MODULE(name,enc,senc) \
	{#name, enc, name##_##senc##_create_env, name##_##senc##_close_env, name##_##senc##_stem}

static const stemmer_module stemmer_modules[] =
{
	/*
	 * 来自Snowball发行版的词干提取器列表
	 */
	STEMMER_MODULE(basque, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(catalan, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(danish, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(dutch, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(english, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(finnish, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(french, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(german, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(indonesian, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(irish, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(italian, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(norwegian, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(porter, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(portuguese, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(spanish, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(swedish, PG_LATIN1, ISO_8859_1),
	STEMMER_MODULE(hungarian, PG_LATIN2, ISO_8859_2),
	STEMMER_MODULE(romanian, PG_LATIN2, ISO_8859_2),
	STEMMER_MODULE(russian, PG_KOI8R, KOI8_R),
	STEMMER_MODULE(arabic, PG_UTF8, UTF_8),
	STEMMER_MODULE(armenian, PG_UTF8, UTF_8),
	STEMMER_MODULE(basque, PG_UTF8, UTF_8),
	STEMMER_MODULE(catalan, PG_UTF8, UTF_8),
	STEMMER_MODULE(danish, PG_UTF8, UTF_8),
	STEMMER_MODULE(dutch, PG_UTF8, UTF_8),
	STEMMER_MODULE(english, PG_UTF8, UTF_8),
	STEMMER_MODULE(finnish, PG_UTF8, UTF_8),
	STEMMER_MODULE(french, PG_UTF8, UTF_8),
	STEMMER_MODULE(german, PG_UTF8, UTF_8),
	STEMMER_MODULE(greek, PG_UTF8, UTF_8),
	STEMMER_MODULE(hindi, PG_UTF8, UTF_8),
	STEMMER_MODULE(hungarian, PG_UTF8, UTF_8),
	STEMMER_MODULE(indonesian, PG_UTF8, UTF_8),
	STEMMER_MODULE(irish, PG_UTF8, UTF_8),
	STEMMER_MODULE(italian, PG_UTF8, UTF_8),
	STEMMER_MODULE(lithuanian, PG_UTF8, UTF_8),
	STEMMER_MODULE(nepali, PG_UTF8, UTF_8),
	STEMMER_MODULE(norwegian, PG_UTF8, UTF_8),
	STEMMER_MODULE(porter, PG_UTF8, UTF_8),
	STEMMER_MODULE(portuguese, PG_UTF8, UTF_8),
	STEMMER_MODULE(romanian, PG_UTF8, UTF_8),
	STEMMER_MODULE(russian, PG_UTF8, UTF_8),
	STEMMER_MODULE(serbian, PG_UTF8, UTF_8),
	STEMMER_MODULE(spanish, PG_UTF8, UTF_8),
	STEMMER_MODULE(swedish, PG_UTF8, UTF_8),
	STEMMER_MODULE(tamil, PG_UTF8, UTF_8),
	STEMMER_MODULE(turkish, PG_UTF8, UTF_8),
	STEMMER_MODULE(yiddish, PG_UTF8, UTF_8),

	/*
	 * PG_SQL_ASCII编码的词干提取器应适用于任何服务器
	 * 编码
	 */
	STEMMER_MODULE(english, PG_SQL_ASCII, ISO_8859_1),

	{NULL, 0, NULL, NULL, NULL} /* 列表结束标记 */
};


typedef struct DictSnowball
{
	struct SN_env *z;
	StopList	stoplist;
	bool		needrecode;		/* 在调用词干提取之前/之后需要重新编码 */
	int			(*stem) (struct SN_env *z);

	/*
	 * Snowball在调用之间保存分配的内存，因此我们应该在我们的
	 * 私有内存上下文中运行它。注意，初始化函数在长生命周期
	 * 上下文中执行，因此我们只需记住CurrentMemoryContext
	 */
	MemoryContext dictCtx;
} DictSnowball;


static void fc_locate_stem_module(DictSnowball *fc_d, const char *fc_lang)
{
	const stemmer_module *fc_m;

	/*
	 * 首先，尝试找到词干提取模块的精确匹配。PG_SQL_ASCII编码的
	 * 词干提取器被视为与任何服务器编码一起工作
	 */
	for (fc_m = stemmer_modules; fc_m->name; fc_m++)
	{
		if ((fc_m->enc == PG_SQL_ASCII || fc_m->enc == GetDatabaseEncoding()) &&
			pg_strcasecmp(fc_m->name, fc_lang) == 0)
		{
			fc_d->stem = fc_m->stem;
			fc_d->z = fc_m->create();
			fc_d->needrecode = false;
			return;
		}
	}

	/*
	 * 其次，尝试为所需语言找到UTF8编码的词干提取器。
	 */
	for (fc_m = stemmer_modules; fc_m->name; fc_m++)
	{
		if (fc_m->enc == PG_UTF8 && pg_strcasecmp(fc_m->name, fc_lang) == 0)
		{
			fc_d->stem = fc_m->stem;
			fc_d->z = fc_m->create();
			fc_d->needrecode = true;
			return;
		}
	}

	ereport(ERROR,
			(errcode(ERRCODE_UNDEFINED_OBJECT),
			 errmsg("no Snowball stemmer available for language \"%s\" and encoding \"%s\"",
					fc_lang, GetDatabaseEncodingName())));
}

Datum dsnowball_init(PG_FUNCTION_ARGS)
{
	List	   *fc_dictoptions = (List *) PG_GETARG_POINTER(0);
	DictSnowball *fc_d;
	bool		fc_stoploaded = false;
	ListCell   *fc_l;

	fc_d = (DictSnowball *) palloc0(sizeof(DictSnowball));

	foreach(fc_l, fc_dictoptions)
	{
		DefElem    *fc_defel = (DefElem *) lfirst(fc_l);

		if (strcmp(fc_defel->defname, "stopwords") == 0)
		{
			if (fc_stoploaded)
				ereport(ERROR,
						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
						 errmsg("multiple StopWords parameters")));
			readstoplist(defGetString(fc_defel), &fc_d->stoplist, lowerstr);
			fc_stoploaded = true;
		}
		else if (strcmp(fc_defel->defname, "language") == 0)
		{
			if (fc_d->stem)
				ereport(ERROR,
						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
						 errmsg("multiple Language parameters")));
			fc_locate_stem_module(fc_d, defGetString(fc_defel));
		}
		else
		{
			ereport(ERROR,
					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
					 errmsg("unrecognized Snowball parameter: \"%s\"",
							fc_defel->defname)));
		}
	}

	if (!fc_d->stem)
		ereport(ERROR,
				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
				 errmsg("missing Language parameter")));

	fc_d->dictCtx = CurrentMemoryContext;

	PG_RETURN_POINTER(fc_d);
}

Datum dsnowball_lexize(PG_FUNCTION_ARGS)
{
	DictSnowball *fc_d = (DictSnowball *) PG_GETARG_POINTER(0);
	char	   *fc_in = (char *) PG_GETARG_POINTER(1);
	int32		fc_len = PG_GETARG_INT32(2);
	char	   *fc_txt = lowerstr_with_len(fc_in, fc_len);
	TSLexeme   *fc_res = palloc0(sizeof(TSLexeme) * 2);

	/*
	 * 不要将超过1000字节的字符串传递给词干提取器，因为它们
	 * 肯定不是任何人类语言中的单词。此限制避免了在诸如
	 * base64编码数据之类的内容上浪费周期，并保护了我们
	 * 免受词干提取器可能出现的效率低下或错误行为的影响。
	 * （例如，土耳其语词干提取器具有无限递归，因此在足够
	 * 长的字符串上可能会崩溃。）然而，Snowball字典定义为识别
	 * 所有字符串，因此我们无法将字符串拒绝为未知单词。
	 */
	if (fc_len > 1000)
	{
		/* 返回词元小写，但其他未修改 */
		fc_res->lexeme = fc_txt;
	}
	else if (*fc_txt == '\0' || searchstoplist(&(fc_d->stoplist), fc_txt))
	{
		/* 为空或停用词，因此报告为停用词 */
		pfree(fc_txt);
	}
	else
	{
		MemoryContext fc_saveCtx;

		/*
		 * 如果词干提取器是utf8并且不匹配服务器编码，则重编码为utf8
		 */
		if (fc_d->needrecode)
		{
			char	   *fc_recoded;

			fc_recoded = pg_server_to_any(fc_txt, strlen(fc_txt), PG_UTF8);
			if (fc_recoded != fc_txt)
			{
				pfree(fc_txt);
				fc_txt = fc_recoded;
			}
		}

		/* 查看关于d->dictCtx的注释 */
		fc_saveCtx = MemoryContextSwitchTo(fc_d->dictCtx);
		SN_set_current(fc_d->z, strlen(fc_txt), (symbol *) fc_txt);
		fc_d->stem(fc_d->z);
		MemoryContextSwitchTo(fc_saveCtx);

		if (fc_d->z->p && fc_d->z->l)
		{
			fc_txt = repalloc(fc_txt, fc_d->z->l + 1);
			memcpy(fc_txt, fc_d->z->p, fc_d->z->l);
			fc_txt[fc_d->z->l] = '\0';
		}

		/* 如果需要，进行反向编码 */
		if (fc_d->needrecode)
		{
			char	   *fc_recoded;

			fc_recoded = pg_any_to_server(fc_txt, strlen(fc_txt), PG_UTF8);
			if (fc_recoded != fc_txt)
			{
				pfree(fc_txt);
				fc_txt = fc_recoded;
			}
		}

		fc_res->lexeme = fc_txt;
	}

	PG_RETURN_POINTER(fc_res);
}
